package com.payne;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

/**
 * Hello world!
 *
 */
public class App implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
    private static int count =0;

    private static String url="http://cnbtkitty.co/search/e7Kj62n_kqe921_s3WIKAA/1/0/0.html";

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {


        if(url.equals(page.getUrl().toString())){
            //aa
            List<String> all = page.getHtml().xpath("//div[@class=\"list-box\"]/dl/dt/a/@href").all();
            for (String s : all) {
                //http://cnbtkitty.co/t/BcEJAQAgCAPASuCDI46w2T-Cd3YIcI3o8DqWKD5eylumxm4jc6Y-.html
                ///t/BcEJAQAgCAPASuCDI46w2T-Cd3YIcI3o8DqWKD5eylumxm4jc6Y-.html
                System.out.println("http://cnbtkitty.co"+s);

            }
            page.addTargetRequests(all);
        }else {
            System.out.println(page.getHtml().xpath("//dd[@class=\"magnet\"]/a/text()").get());
        }

    }
//bb
    public static void main(String[] args) {
        Spider.create(new App()).addUrl(url).thread(5).run();
    }
}
